{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from multiprocessing import Process\n",
    "from multiprocessing import Pool\n",
    "import os\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from lxml import etree\n",
    "import pandas as pd\n",
    "import openpyxl\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def group_url():\n",
    "    url = 'http://www.boohee.com/food/'\n",
    "    res = requests.get(url)\n",
    "    html = res.text\n",
    "    bs = BeautifulSoup(html,\"html.parser\")\n",
    "      #装每种食物种类的url\n",
    "    li = bs.find(class_='row').find_all('li')  #每种食物大类的url\n",
    "    for f in li:  #f代表每种食物大类\n",
    "        group_url = f.find(class_=\"img-box\").find('a')['href']\n",
    "        group_url = 'http://www.boohee.com'+group_url\n",
    "        yield group_url    #group_url是每一种食物大类的url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def book_url(group_url): \n",
    "    urls = []   #存储每类食物中下的翻页\n",
    "    for i in range(1,11):\n",
    "        page_url = group_url+'?page='+str(i)\n",
    "        urls.append(page_url)\n",
    "        # urls存储的是每一类食物中的所有页\n",
    "    return urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_info(urls):    \n",
    "    data = []\n",
    "    food_end = []\n",
    "    pool = Pool(processes=4)\n",
    "    info = []\n",
    "    info_url = []    #食物详情的url\n",
    "    s = pool.map(requests.get,urls)\n",
    "    for i in s:\n",
    "        html = i.text\n",
    "        bs = BeautifulSoup(html,\"html.parser\")\n",
    "        group = bs.find(class_='widget-food-list pull-right').find('h3').text    #食物类别\n",
    "        end = bs.find_all(class_='img-box pull-left')\n",
    "        for j in end:\n",
    "            info_url.append('http://www.boohee.com{0}'.format(j.find('a')['href']))\n",
    "        \n",
    "    data.append(pool.map(get_detil,info_url))\n",
    "    frame = pd.DataFrame(data=data[0] ,columns=['名称' ,'又名' ,'热量（大卡）' ,'碳水化合物（g）' ,'脂肪（g）' ,'蛋白质（g）' ,'建议'])\n",
    "    return group ,frame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_detil(url):\n",
    "    # 获取详情页面的具体信息\n",
    "    print(url)\n",
    "    s = requests.get(url = url)\n",
    "    if s.status_code != 200:\n",
    "        N = ('', '', '', '', '', '', '')\n",
    "        return N\n",
    "    text = s.text\n",
    "    html = etree.HTML(text)\n",
    "    try:\n",
    "        name = html.xpath('/html/body/div[1]/div[3]/div/h2/a[2]')[0].tail   #名字\n",
    "    except Exception as e:\n",
    "        name = ''\n",
    "    other_name = html.xpath('/html/body/div[1]/div[3]/div/div[2]/div[1]/div/ul/li[1]/b')[0].tail    #别名\n",
    "    calories = html.xpath('/html/body/div[1]/div[3]/div/div[2]/div[2]/div/dl[2]/dd[1]/span[2]/span')[0].text    #热量\n",
    "    carbohydrate = html.xpath('/html/body/div[1]/div[3]/div/div[2]/div[2]/div/dl[2]/dd[2]/span[2]')[0].text    #碳水化合物\n",
    "    adipose = html.xpath('/html/body/div[1]/div[3]/div/div[2]/div[2]/div/dl[3]/dd[1]/span[2]')[0].text    #脂肪\n",
    "    protein = html.xpath('/html/body/div[1]/div[3]/div/div[2]/div[2]/div/dl[3]/dd[2]/span[2]')[0].text    #蛋白质\n",
    "    suggestion = html.xpath('/html/body/div[1]/div[3]/div/div[2]/div[1]/p/b')[0].tail    #建议\n",
    "    return name ,other_name ,calories ,carbohydrate ,adipose ,protein ,suggestion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_excel(data,sheet_name):\n",
    "    writer = pd.ExcelWriter('Excel_test.xlsx',engin='openpyxl')\n",
    "    book = openpyxl.load_workbook(writer.path)\n",
    "    writer.book = book\n",
    "    df = data\n",
    "    df.to_excel(excel_writer=writer,sheet_name=sheet_name)\n",
    "    writer.save()\n",
    "    writer.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.12702536582946777\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "url = group_url()\n",
    "for i in url:\n",
    "    r = book_url(group_url=i)\n",
    "    e = get_info(urls=r)\n",
    "    save_excel(data=e[1],sheet_name=e[0])\n",
    "end_time = time.time()\n",
    "print(end_time - start_time)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
